{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Week 8 特征工程 作业二"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 输入数据，数据准备与处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 导入数据函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#--------------------------\n",
    "# 导入数据函数\n",
    "#--------------------------\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def load_housing_data(housing_path = './'):\n",
    "    csv_path = os.path.join(housing_path, 'housing.csv')\n",
    "    return pd.read_csv(csv_path)\n",
    "\n",
    "housing = load_housing_data()\n",
    "\n",
    "# 按照 income一列的数值分层（5类）\n",
    "# pd.cut()的作用，是把连续值转换成类别标签 ，加入housing中新建一列\n",
    "housing['income_cat'] = pd.cut(housing['median_income'],bins = [0., 1.5, 3.0, 4.5, 6., np.inf], labels = [1,2,3,4,5])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 分层分割训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#--------------------------\n",
    "# 按照 income 一列的数值分层分割训练集和测试集（80%：20%）\n",
    "#--------------------------\n",
    "from sklearn.model_selection import StratifiedShuffleSplit  \n",
    "\n",
    "# 函数先将样本随机打乱，然后根据设置参数划分出指定数量的独立的train/test数据集。\n",
    "split  = StratifiedShuffleSplit(n_splits = 1, test_size = 0.2, random_state  = 42)\n",
    "\n",
    "for train_index, test_index in split.split(housing, housing['income_cat']):\n",
    "    strat_train_set = housing.loc[train_index]  # 训练集\n",
    "    strat_test_set = housing.loc[test_index]    # 测试集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 机器学习算法的数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#--------------------------    \n",
    "# 机器学习算法的数据准备  results： housing_labels  vs  housing_final \n",
    "#--------------------------\n",
    "# 标签列\n",
    "housing_labels = strat_train_set['median_house_value'].copy()\n",
    "\n",
    "# 数据列\n",
    "housing1 = strat_train_set.drop(['median_house_value', 'income_cat'], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 自定义转换器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 转换器\n",
    "from sklearn.base import BaseEstimator, TransformerMixin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 根据列名选择DataFrame中的列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    # 根据列名选择DataFrame中的列\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names].values  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 增加组合列 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rooms_ix, bedrooms_ix, population_ix, household_ix = [list(housing1.columns).index(col) for col in ('total_rooms','total_bedrooms','population','households')]\n",
    "\n",
    "# 构造的附加组合列列名\n",
    "extra_attribs = [\"rooms_per_household\", \"population_per_household\", \"bedrooms_per_room\"]\n",
    "\n",
    "class CombinedAttributesAdder(BaseEstimator, TransformerMixin):\n",
    "    # 增加附加组合列 \n",
    "    def __init__(self, add_bedrooms_per_room = True):\n",
    "        self.add_bedrooms_per_room = add_bedrooms_per_room\n",
    "        \n",
    "    def fit(self, X, y = None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y = None):\n",
    "        rooms_per_household = X[:,rooms_ix] / X[:,household_ix]\n",
    "        population_per_household = X[:,population_ix] / X[:,household_ix]\n",
    "        \n",
    "        if self.add_bedrooms_per_room:\n",
    "            bedrooms_per_room = X[:,bedrooms_ix]/X[:,rooms_ix]\n",
    "            # np.c_()函数将多个DataFrame，Series对象合并成一个\n",
    "            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]   \n",
    "        else:\n",
    "            return np.c_[X, rooms_per_household, population_per_household]   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 将文本数据列转换成独热编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_one_hot_attribs = []\n",
    "class MyLabelBinarizer(TransformerMixin):\n",
    "    # 将文本数据列转换成独热编码\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        self.encoder = LabelBinarizer(*args, **kwargs)\n",
    "    def fit(self, x, y=0):\n",
    "        self.encoder.fit(x)\n",
    "        return self\n",
    "    def transform(self, x, y=0):\n",
    "        global cat_one_hot_attribs \n",
    "        cat_one_hot_attribs = list(self.encoder.classes_) \n",
    "        return self.encoder.transform(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 转换流水线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "      <th>rooms_per_household</th>\n",
       "      <th>population_per_household</th>\n",
       "      <th>bedrooms_per_room</th>\n",
       "      <th>&lt;1H OCEAN</th>\n",
       "      <th>INLAND</th>\n",
       "      <th>ISLAND</th>\n",
       "      <th>NEAR BAY</th>\n",
       "      <th>NEAR OCEAN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17606</th>\n",
       "      <td>-1.156043</td>\n",
       "      <td>0.771950</td>\n",
       "      <td>0.743331</td>\n",
       "      <td>-0.493234</td>\n",
       "      <td>-0.445438</td>\n",
       "      <td>-0.636211</td>\n",
       "      <td>-0.420698</td>\n",
       "      <td>-0.614937</td>\n",
       "      <td>-0.312055</td>\n",
       "      <td>-0.086499</td>\n",
       "      <td>0.155318</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18632</th>\n",
       "      <td>-1.176025</td>\n",
       "      <td>0.659695</td>\n",
       "      <td>-1.165317</td>\n",
       "      <td>-0.908967</td>\n",
       "      <td>-1.036928</td>\n",
       "      <td>-0.998331</td>\n",
       "      <td>-1.022227</td>\n",
       "      <td>1.336459</td>\n",
       "      <td>0.217683</td>\n",
       "      <td>-0.033534</td>\n",
       "      <td>-0.836289</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14650</th>\n",
       "      <td>1.186849</td>\n",
       "      <td>-1.342183</td>\n",
       "      <td>0.186642</td>\n",
       "      <td>-0.313660</td>\n",
       "      <td>-0.153345</td>\n",
       "      <td>-0.433639</td>\n",
       "      <td>-0.093318</td>\n",
       "      <td>-0.532046</td>\n",
       "      <td>-0.465315</td>\n",
       "      <td>-0.092405</td>\n",
       "      <td>0.422200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>-0.017068</td>\n",
       "      <td>0.313576</td>\n",
       "      <td>-0.290520</td>\n",
       "      <td>-0.362762</td>\n",
       "      <td>-0.396756</td>\n",
       "      <td>0.036041</td>\n",
       "      <td>-0.383436</td>\n",
       "      <td>-1.045566</td>\n",
       "      <td>-0.079661</td>\n",
       "      <td>0.089736</td>\n",
       "      <td>-0.196453</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3555</th>\n",
       "      <td>0.492474</td>\n",
       "      <td>-0.659299</td>\n",
       "      <td>-0.926736</td>\n",
       "      <td>1.856193</td>\n",
       "      <td>2.412211</td>\n",
       "      <td>2.724154</td>\n",
       "      <td>2.570975</td>\n",
       "      <td>-0.441437</td>\n",
       "      <td>-0.357834</td>\n",
       "      <td>-0.004194</td>\n",
       "      <td>0.269928</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       longitude  latitude  housing_median_age  total_rooms  total_bedrooms  \\\n",
       "17606  -1.156043  0.771950            0.743331    -0.493234       -0.445438   \n",
       "18632  -1.176025  0.659695           -1.165317    -0.908967       -1.036928   \n",
       "14650   1.186849 -1.342183            0.186642    -0.313660       -0.153345   \n",
       "3230   -0.017068  0.313576           -0.290520    -0.362762       -0.396756   \n",
       "3555    0.492474 -0.659299           -0.926736     1.856193        2.412211   \n",
       "\n",
       "       population  households  median_income  rooms_per_household  \\\n",
       "17606   -0.636211   -0.420698      -0.614937            -0.312055   \n",
       "18632   -0.998331   -1.022227       1.336459             0.217683   \n",
       "14650   -0.433639   -0.093318      -0.532046            -0.465315   \n",
       "3230     0.036041   -0.383436      -1.045566            -0.079661   \n",
       "3555     2.724154    2.570975      -0.441437            -0.357834   \n",
       "\n",
       "       population_per_household  bedrooms_per_room  <1H OCEAN  INLAND  ISLAND  \\\n",
       "17606                 -0.086499           0.155318        1.0     0.0     0.0   \n",
       "18632                 -0.033534          -0.836289        1.0     0.0     0.0   \n",
       "14650                 -0.092405           0.422200        0.0     0.0     0.0   \n",
       "3230                   0.089736          -0.196453        0.0     1.0     0.0   \n",
       "3555                  -0.004194           0.269928        1.0     0.0     0.0   \n",
       "\n",
       "       NEAR BAY  NEAR OCEAN  \n",
       "17606       0.0         0.0  \n",
       "18632       0.0         0.0  \n",
       "14650       0.0         1.0  \n",
       "3230        0.0         0.0  \n",
       "3555        0.0         0.0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 转换流水线\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# 数值列流水线\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "# 对数值列作处理的Pipeline\n",
    "housing_num = housing1.drop('ocean_proximity', axis = 1)\n",
    "num_attribs = list(housing_num) \n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "        #名称/转换器\n",
    "        ('selector', DataFrameSelector(num_attribs)),\n",
    "        ('imputer', SimpleImputer(strategy=\"median\")),\n",
    "        ('attribs_adder', CombinedAttributesAdder()),\n",
    "        ('std_scaler', StandardScaler()),\n",
    "    ])\n",
    "\n",
    "# 文字列流水线\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "# 对文字列作处理的Pipeline\n",
    "cat_attribs = ['ocean_proximity']\n",
    "#cat_one_hot_attribs = list(encoder.classes_)\n",
    "\n",
    "cat_pipeline = Pipeline([\n",
    "        ('selector', DataFrameSelector(cat_attribs)),               \n",
    "        ('LabelBinarizer', MyLabelBinarizer()),\n",
    "    ])\n",
    "\n",
    "\n",
    "# 组合多个流水线\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "full_pipeline = FeatureUnion(transformer_list=[\n",
    "        ('num_pipline', num_pipeline,),\n",
    "        ('cat_pipline', cat_pipeline),\n",
    "    ])\n",
    "\n",
    "# 转换流水线处理数据\n",
    "housing_cat = housing1['ocean_proximity']\n",
    "cat_attribs = ['ocean_proximity']\n",
    "\n",
    "housing_pl = full_pipeline.fit_transform(housing1)\n",
    "\n",
    "# 将上述所有列名组合成一个列名列表（最终数据的每列列名）\n",
    "attributes = num_attribs + extra_attribs + cat_one_hot_attribs\n",
    "\n",
    "housing_tr = pd.DataFrame(housing_pl, index = housing1.index, columns = attributes)\n",
    "housing_tr.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 根据皮尔逊相关系数挑选重要特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选出特征相关性最大的k项特征 \n",
    "class ImportantAttrSelector(BaseEstimator, TransformerMixin):\n",
    "    # 根据皮尔逊相关系数挑选重要特征\n",
    "    def __init__(self, k):\n",
    "        self.k = k\n",
    "        \n",
    "    def fit(self, X_df, y = None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self,X_df, y):\n",
    "        corr_matrix = pd.concat([X_df, y], axis = 1).corr()\n",
    "        corr_ser = corr_matrix['median_house_value']\n",
    "        corr_ser2 = pd.concat([np.abs(corr_ser),corr_ser], axis = 1)\n",
    "        corr_ser2.columns = ['abs', 'corr']\n",
    "        important_attr = corr_ser2.sort_values(by = 'abs', axis = 0, ascending = False)\n",
    "        \n",
    "        return important_attr.iloc[1:self.k+1, 1]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "median_income        0.687160\n",
       "INLAND              -0.482886\n",
       "<1H OCEAN            0.259521\n",
       "bedrooms_per_room   -0.234240\n",
       "NEAR BAY             0.158733\n",
       "Name: corr, dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 选择重要特征\n",
    "importantAttr = ImportantAttrSelector(k = 5)\n",
    "importantAttr.fit(housing_tr)\n",
    "important_attr = importantAttr.transform(housing_tr, housing_labels)\n",
    "#housing_final = pd.DataFrame(housing_pl)\n",
    "#housing_final.head()\n",
    "important_attr"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
